import asyncio
from pyppeteer import launch
from pyppeteer import launcher
import ssl
import random
from concurrent.futures import ThreadPoolExecutor,ProcessPoolExecutor
import datetime
import time
from baidu_sql import baidu_etl_db
from insert_sql import insert_etl_db
from write_baidu_file import baidu_write_xiaofei_info
import subprocess
from func_timeout import func_set_timeout
import timeout_decorator
import functools


tmp_store_baidu_cost_info_dir = 'baidu_xiaofei_info_dir'
ssl._create_default_https_context = ssl._create_unverified_context


yesterday = (datetime.datetime.now() + datetime.timedelta(days=-1)).strftime('%Y%m%d')

class pyppeteer_crawl_baidu:

    def __init__(self, name_password_dict, big_channel):

        self.name_password_dict = name_password_dict
        self.login_url ='https://cas.baidu.com/?tpl=www2&fromu=http%3A%2F%2Fwww2.baidu.com%2Fcommon%2Fappinit.ajax'
        self.big_channel = big_channel

    async def crawl(self,semaphore,url,name,password):
        async with semaphore:
            try:

                browser = await launch(headless=True,executablePath='/bin/chromium-browser',autoClose=True, ignoreDefaultArgs=['--enable-automation'])
                page = await browser.newPage()
                #await page.setViewport(viewport={'width': 1920, 'height': 1440})
                await page.evaluateOnNewDocument('() =>{ Object.defineProperties(navigator,'
                               '{ webdriver:{ get: () => false } }) }')

                await asyncio.wait_for(page.goto(url), timeout=50)

                #title = await page.title()
                #print(title)
                await page.click('#app > div.inner-wrap > div.row.slider-login > div.login.pull-right > div.login-header > span')
                await asyncio.sleep(round(random.uniform(0.71, 1.59), 1))
                await page.type('#uc-common-account',name,{'delay': random.randint(10, 30)})
                #await page.waitForSelector('#uc-common-account',name,{'timeout': 3000})
                await page.type('#ucsl-password-edit',password,{'delay': random.randint(10, 30)})
                await asyncio.wait([page.waitForNavigation({'timeout': 1000*30}),page.click('#submit-form')])
                await asyncio.sleep(round(random.uniform(1.71, 2.59), 1))
                #print(await browser.pages())
                #print(page.url)
                await page.goto(page.url, {'timeout': 1000 * 44})
                await page.evaluate("""{window.scrollBy(0, document.body.scrollHeight);}""")
                #title = await page.title()
                time.sleep(round(random.uniform(1.01, 1.89), 1))
                # 日期选择按钮
                click_handle = await page.xpath('//*[@id="root-container"]/div/div[2]/div[1]/div[2]/div[1]/div[2]/div[2]/div/button')
                await click_handle[0].click()
                time.sleep(round(random.uniform(1.01, 1.89), 1))
                #昨天
                click_handle = await page.xpath('/html/body/div[3]/div/div/div/div/div[1]/div[2]')
                await click_handle[0].click()
                time.sleep(round(random.uniform(1.11, 1.89), 1))
                # 日期
                date_html_elements = await page.xpath('//*[@id="root-container"]/div/div[2]/div[1]/div[2]/div[1]/div[2]/div[2]/div/button/span/span')
                #all_info=[]
                for item in date_html_elements:
                    yesterday_date = await (await item.getProperty('textContent')).jsonValue()
                    #all_info.append(yesterday_date)
                # 账号
                user_html_elements = await page.xpath('//*[@id="root-container"]/div/div[2]/div[2]/div[1]/div/div[1]/span[2]')
                for item in user_html_elements:
                    user = await (await item.getProperty('textContent')).jsonValue()
                    #all_info.append(user)
                # 消费
                xiaofei_html_elements = await page.xpath('//*[@id="root-container"]/div/div[2]/div[1]/div[2]/div[2]/div[1]/div/div[2]')
                for item in xiaofei_html_elements:
                    xiaofei = await (await item.getProperty('textContent')).jsonValue()
                print(page.url, '昨天日期：', yesterday_date,'当前账户：', user,'消费：',xiaofei)
                #print(datetime.datetime.strptime(yesterday_date, '%Y-%m-%d').strftime('%Y%m%d'), xiaofei, {name,password}, self.big_channel)
                if xiaofei:
                    #baidu_etl_db(datetime.datetime.strptime(yesterday_date, '%Y-%m-%d').strftime('%Y%m%d'), '"'+str(round(float(xiaofei.replace(',',''))*100))+'"', '{"'+name+'":"'+password+'"}', self.big_channel).insert_mysql()
                    baidu_write_xiaofei_info(datetime.datetime.strptime(yesterday_date, '%Y-%m-%d').strftime('%Y%m%d')+'\t'+str(round(float(xiaofei.replace(',',''))*100))+'\t{"'+name+'":"'+password+'"}\t'+self.big_channel, self.big_channel, '', './baidu_xiaofei_info_dir/').write_file()
            except Exception as e:
                print(datetime.datetime.now(), "\n!!!!!!!!!\n\t\t",e, name, password)
            finally:
                await browser.close()

    async def main(self):

        semaphore = asyncio.Semaphore(4)
        tasks = [
            self.crawl(semaphore,self.login_url,name,password) for name,password in self.name_password_dict.items()
        ]
        await asyncio.wait(tasks, timeout=88)
        # await asyncio.gather(*tasks)

    @staticmethod
    def offical(name_password_dict, big_channel):

        #asyncio.get_event_loop().run_until_complete(pyppeteer_crawl_baidu(name_password_dict, big_channel).main())
        loop = asyncio.new_event_loop()
        asyncio.set_event_loop(loop)
        loop.run_until_complete(pyppeteer_crawl_baidu(name_password_dict, big_channel).main())

        time_num = 15
        while True:
            orgin_name_password_list = []
            for k, v in name_password_dict.items():
                orgin_name_password_list.append('{"' + k + '":"' + v + '"}')
            miss = list(set(orgin_name_password_list).difference(set(baidu_write_xiaofei_info('', big_channel,'', './baidu_xiaofei_info_dir/').read_name_password())))
            if miss:
                print ('after ', time_num-15 ,' : miss---\t', miss)
                miss_dict_all={}
                for single_dict_str in miss:
                    miss_dict_all.update(eval(single_dict_str))
                time.sleep(time_num)
                loop = asyncio.new_event_loop()
                asyncio.set_event_loop(loop)
                loop.run_until_complete(pyppeteer_crawl_baidu(miss_dict_all, big_channel).main())
                if time_num == 60:
                    break
                time_num += 15
            else:
                if big_channel == 'xinpu' or big_channel =='zhiwei':
                    ys_noys=1.39
                else:
                    ys_noys=1.41
                sum_xiaofei = baidu_write_xiaofei_info('', big_channel, ys_noys, './baidu_xiaofei_info_dir/').read_cost()
                print(sum_xiaofei)
                if sum_xiaofei:
                    insert_etl_db(str(sum_xiaofei), 'toufang_sembaidu', big_channel).insert_mysql()
                break
                
def timeout_exec(func):
    @functools.wraps(func)
    def wrpper(*args, **kw):
        try:
            func
            return func(*args, **kw)
        except:
            try:
                func
                return func(*args, **kw)
            except:
                pass
    return wrpper

@timeout_exec
@timeout_decorator.timeout(360, use_signals=True)
def xinpu():

    name_password_dict={
                        'user-crawler-人保1BQWEG10086':'passwd',
                        'user-crawler-人保2BQWEG10086':'passwd',
                        'user-crawler-人保3BQWEG10086':'passwd',
                        'user-crawler-人保4BQWEG10086':'passwd',
                        'user-crawler-人保5BQWEG10086':'passwd',
                        'user-crawler-泰康-BQWEG10086':'passwd',
                        'user-crawler-泰康1BQWEG10086':'passwd',
                        'user-crawler-泰康2BQWEG10086':'passwd',
                        'user-crawler-泰康3BQWEG10086':'passwd',
                        'user-crawler-华泰1BQWEG10086':'passwd',
                        'user-crawler-安盛1BQWEG10086':'passwd',
                        'user-国寿1C21KA10086':'passwd',
                        'user-国寿2C21KA10086':'passwd',
                        'user-国寿3C21KA10086':'passwd',
                        'user-国寿4C21KA10086':'passwd',
                        'user-crawler-人保6BQWEG10086':'passwd',
                        'user-crawler-人保7BQWEG10086':'passwd',
                        'user-crawler-人保8BQWEG10086':'passwd',
                        'user-crawler-人保9BQWEG10086':'passwd',
                        'user-人保10BQWEG10086':'passwd',
                        'user-人保11BQWEG10086':'passwd'
                        }
    subprocess.call("rm -f {}/{}_data".format(tmp_store_baidu_cost_info_dir, (datetime.datetime.now() + datetime.timedelta(days=-1)).strftime('%Y-%m-%d') + '/xinpu'), shell=True)
    pyppeteer_crawl_baidu.offical(name_password_dict, big_channel='xinpu')

@timeout_exec
@timeout_decorator.timeout(360, use_signals=True)
def xinpuys():

    name_password_dict={
                        'useryuan1-BQWEG00669': 'passwd',
                        'useryuan2-BQWEG00669': 'passwd',
                        'useryuan3-BQWEG00669': 'passwd',
                        'useryuan4-BQWEG00669': 'passwd',
                        'useryuan5-BQWEG00669': 'passwd',
                        'useryuan7-BQWEG00669': 'passwd',
                        'useryuan8-BQWEG00669': 'passwd',
                        'useryuan9-BQWEG00669': 'passwd',
                        'useryuan10-BQWEG00669': 'passwd',
                        'useryuan11-BQWEG00669': 'passwd',
                        'useryuan12-BQWEG00669': 'passwd',
                        'useryuan13-BQWEG00669': 'passwd',
                        'useryuan14-BQWEG00669': 'passwd',
                        'useryuan15-BQWEG00669': 'passwd',
                        'useryuan16-BQWEG00669': 'passwd',
                        'useryuan-华泰1BQWEG00669': 'passwd',
                        'useryuan-华泰2BQWEG00669': 'passwd',
                        'useryuanw1-BQWEG00669': 'passwd',
                        'useryuanw2-BQWEG00669': 'passwd',
                        'useryuanw6-BQWEG00669': 'passwd',
                        'useryuan22-BQWEG00669': 'passwd',
                        'useryuan23-BQWEG00669': 'passwd',
                        'useryuan24-BQWEG00669': 'passwd',
                        'useryuan25-BQWEG00669': 'passwd',
                        'useryuan27-BQWEG00669': 'passwd'
                       }
    subprocess.call("rm -f {}/{}_data".format(tmp_store_baidu_cost_info_dir, (datetime.datetime.now() + datetime.timedelta(days=-1)).strftime('%Y-%m-%d') + '/xinpuys'), shell=True)
    pyppeteer_crawl_baidu.offical(name_password_dict, big_channel='xinpuys')

@timeout_exec
@timeout_decorator.timeout(360, use_signals=True)
def zhiwei():

    name_password_dict={
                       'userzhiwei1-QWER06180':'passwd',
                       'userzhiwei2-QWER06180':'passwd',
                       'userzhiwei3-QWER06180':'passwd',
                       'userzhiwei4-QWER06180':'passwd',
                       'userzhiwei5-QWER06180':'passwd'
                       }
    subprocess.call("rm -f {}/{}_data".format(tmp_store_baidu_cost_info_dir, (datetime.datetime.now() + datetime.timedelta(days=-1)).strftime('%Y-%m-%d') + '/zhiwei'), shell=True)
    pyppeteer_crawl_baidu.offical(name_password_dict, big_channel='zhiwei')

@timeout_exec
@timeout_decorator.timeout(360, use_signals=True)
def zhiweiys():

    name_password_dict={
                       'useryuanzhi1-UYTR806018':'passwd',
                       'useryuanzhi2-UYTR806018':'passwd',
                       'useryuanzhi3-UYTR806018':'passwd',
                       'useryuanzhi4-UYTR806018':'passwd',
                       'useryuanzhi5-UYTR806018':'passwd',
                       'useryuanzhi11-KUYW806018':'passwd',
                       'useryuanzhi12-KUYW806018':'passwd',
                       'useryuanzhi13-KUYW806018':'passwd',
                       'useryuanzhi14-KUYW806018':'passwd',
                       'useryuanzhi15-KUYW806018':'passwd'
                        }
    subprocess.call("rm -f {}/{}_data".format(tmp_store_baidu_cost_info_dir, (datetime.datetime.now() + datetime.timedelta(days=-1)).strftime('%Y-%m-%d') + '/zhiweiys'), shell=True)
    pyppeteer_crawl_baidu.offical(name_password_dict, big_channel='zhiweiys')


if __name__ == '__main__':

    # xinpu()
    # xinpuys()
    zhiwei()
    # zhiweiys()
